{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import _pickle as cPickle\n",
    "import json\n",
    "from numpy.random import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用户和item索引\n",
    "users_index = cPickle.load(open('ml-100k/users_index.pkl','rb'))\n",
    "items_index = cPickle.load(open('ml-100k/items_index.pkl','rb'))\n",
    "\n",
    "n_users = len(users_index)\n",
    "n_items = len(items_index)\n",
    "\n",
    "# 到排序\n",
    "#每个用户打过分的电影\n",
    "user_items = cPickle.load(open('ml-100k/user_items.pkl','rb'))\n",
    "item_users = cPickle.load(open('ml-100k/item_users.pkl','rb'))\n",
    "\n",
    "# 随机梯度下降会访问样本，直接用原始样本就可以了\n",
    "# 用户关系矩阵\n",
    "# user_item_scores = sio.mmread('ml-100k/user_item_scores')\n",
    "# user_item_scores = user_item_scores.tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating\n",
       "0        1        1       5\n",
       "1        1        2       3\n",
       "2        1        3       4\n",
       "3        1        4       3\n",
       "4        1        5       3"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u_names = ['user_id','item_id','rating','timestamp']\n",
    "u_data = pd.read_csv('ml-100k/u1.base',sep='\\t',encoding='latin-1',names=u_names)\n",
    "u_data = u_data.drop(['timestamp'],axis = 1)\n",
    "u_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化模型参数\n",
    "# 隐含变量的维数K\n",
    "\n",
    "K = 40\n",
    "\n",
    "#隐含变量的偏置\n",
    "# 初始化维x * 1的向量\n",
    "bu = np.zeros((n_users,1))\n",
    "bi = np.zeros((n_items,1))\n",
    "\n",
    "# item和用户的隐含变量\n",
    "pu = np.zeros((n_users,K))\n",
    "qi = np.zeros((n_items,K))\n",
    "\n",
    "# 用随机变量初始化pu和qi\n",
    "for uid in range(n_users):\n",
    "    pu[uid] = np.reshape(random((1,K))/10*(np.sqrt(K)),K)\n",
    "    \n",
    "for iid in range(n_items):\n",
    "    qi[iid] = np.reshape(random((1,K))/10*(np.sqrt(K)),K)\n",
    "\n",
    "mu = u_data['rating'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def svd_pred(uid, iid):\n",
    "    score = mu + bi[iid] + bu[uid] + np.sum(qi[iid] * pu[uid])\n",
    "    \n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 0 -th step is running\n",
      "the rmse of this step on train data is [1.18363899]\n",
      "The 1 -th step is running\n",
      "the rmse of this step on train data is [0.92673266]\n",
      "The 2 -th step is running\n",
      "the rmse of this step on train data is [0.90705088]\n",
      "The 3 -th step is running\n",
      "the rmse of this step on train data is [0.89592161]\n",
      "The 4 -th step is running\n",
      "the rmse of this step on train data is [0.88638511]\n",
      "The 5 -th step is running\n",
      "the rmse of this step on train data is [0.87788839]\n",
      "The 6 -th step is running\n",
      "the rmse of this step on train data is [0.86998113]\n",
      "The 7 -th step is running\n",
      "the rmse of this step on train data is [0.86296938]\n",
      "The 8 -th step is running\n",
      "the rmse of this step on train data is [0.85734315]\n",
      "The 9 -th step is running\n",
      "the rmse of this step on train data is [0.8517987]\n",
      "The 10 -th step is running\n",
      "the rmse of this step on train data is [0.84805611]\n",
      "The 11 -th step is running\n",
      "the rmse of this step on train data is [0.84436956]\n",
      "The 12 -th step is running\n",
      "the rmse of this step on train data is [0.84096368]\n",
      "The 13 -th step is running\n",
      "the rmse of this step on train data is [0.83791638]\n",
      "The 14 -th step is running\n",
      "the rmse of this step on train data is [0.83522383]\n",
      "The 15 -th step is running\n",
      "the rmse of this step on train data is [0.83258185]\n",
      "The 16 -th step is running\n",
      "the rmse of this step on train data is [0.83065393]\n",
      "The 17 -th step is running\n",
      "the rmse of this step on train data is [0.8285283]\n",
      "The 18 -th step is running\n",
      "the rmse of this step on train data is [0.8270345]\n",
      "The 19 -th step is running\n",
      "the rmse of this step on train data is [0.82543798]\n",
      "The 20 -th step is running\n",
      "the rmse of this step on train data is [0.82381158]\n",
      "The 21 -th step is running\n",
      "the rmse of this step on train data is [0.82256665]\n",
      "The 22 -th step is running\n",
      "the rmse of this step on train data is [0.82111695]\n",
      "The 23 -th step is running\n",
      "the rmse of this step on train data is [0.82005573]\n",
      "The 24 -th step is running\n",
      "the rmse of this step on train data is [0.81897266]\n",
      "The 25 -th step is running\n",
      "the rmse of this step on train data is [0.81831798]\n",
      "The 26 -th step is running\n",
      "the rmse of this step on train data is [0.8173431]\n",
      "The 27 -th step is running\n",
      "the rmse of this step on train data is [0.816423]\n",
      "The 28 -th step is running\n",
      "the rmse of this step on train data is [0.81571181]\n",
      "The 29 -th step is running\n",
      "the rmse of this step on train data is [0.81493033]\n",
      "The 30 -th step is running\n",
      "the rmse of this step on train data is [0.81431736]\n",
      "The 31 -th step is running\n",
      "the rmse of this step on train data is [0.81378912]\n",
      "The 32 -th step is running\n",
      "the rmse of this step on train data is [0.81313227]\n",
      "The 33 -th step is running\n",
      "the rmse of this step on train data is [0.81271907]\n",
      "The 34 -th step is running\n",
      "the rmse of this step on train data is [0.812264]\n",
      "The 35 -th step is running\n",
      "the rmse of this step on train data is [0.81178531]\n",
      "The 36 -th step is running\n",
      "the rmse of this step on train data is [0.81140341]\n",
      "The 37 -th step is running\n",
      "the rmse of this step on train data is [0.81105525]\n",
      "The 38 -th step is running\n",
      "the rmse of this step on train data is [0.81066163]\n",
      "The 39 -th step is running\n",
      "the rmse of this step on train data is [0.81032703]\n",
      "The 40 -th step is running\n",
      "the rmse of this step on train data is [0.81004413]\n",
      "The 41 -th step is running\n",
      "the rmse of this step on train data is [0.80977024]\n",
      "The 42 -th step is running\n",
      "the rmse of this step on train data is [0.80948151]\n",
      "The 43 -th step is running\n",
      "the rmse of this step on train data is [0.80923985]\n",
      "The 44 -th step is running\n",
      "the rmse of this step on train data is [0.80901692]\n",
      "The 45 -th step is running\n",
      "the rmse of this step on train data is [0.80879511]\n",
      "The 46 -th step is running\n",
      "the rmse of this step on train data is [0.80860818]\n",
      "The 47 -th step is running\n",
      "the rmse of this step on train data is [0.80841478]\n",
      "The 48 -th step is running\n",
      "the rmse of this step on train data is [0.80825544]\n",
      "The 49 -th step is running\n",
      "the rmse of this step on train data is [0.8081127]\n"
     ]
    }
   ],
   "source": [
    "# 模型训练\n",
    "# gamma 学习率\n",
    "# Lamada 正则参数\n",
    "# steps 迭代次数\n",
    "\n",
    "steps = 50\n",
    "gamma = 0.04\n",
    "Lamada = 0.15\n",
    "\n",
    "# 总的打分记录数目\n",
    "n_records = u_data.shape[0]\n",
    "\n",
    "for step in range(steps):\n",
    "    print('The {} -th step is running'.format(step))\n",
    "    rmse_sum = 0.0\n",
    "    \n",
    "    # 将训练样本打散  获取一个乱序的array\n",
    "    kk = np.random.permutation(n_records)\n",
    "    for j in range(n_records):\n",
    "        line = kk[j]\n",
    "        \n",
    "        uid = users_index[u_data.iloc[line]['user_id']]\n",
    "        iid = items_index[u_data.iloc[line]['item_id']]\n",
    "        \n",
    "        rating = u_data.iloc[line]['rating']\n",
    "        \n",
    "        # 预测残差\n",
    "        eui = rating - svd_pred(uid,iid)\n",
    "        rmse_sum += eui ** 2\n",
    "        \n",
    "        # 随机梯度下降更新\n",
    "        bu[uid] += gamma * (eui - Lamada * bu[uid])\n",
    "        bi[iid] += gamma * (eui - Lamada * bi[iid])\n",
    "        \n",
    "        # 更新qi pu 加了正则项\n",
    "        temp = qi[iid]\n",
    "        qi[iid] += gamma * (eui * pu[uid] - Lamada * qi[iid])\n",
    "        pu[uid] += gamma * (eui * temp - Lamada * pu[uid])\n",
    "        \n",
    "    # 学习率递减\n",
    "    gamma = gamma * 0.93\n",
    "    print('the rmse of this step on train data is {}'.format(np.sqrt(rmse_sum/n_records)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_json(filepath):\n",
    "    dict_ = {}\n",
    "    dict_['mu'] = mu\n",
    "    dict_['K'] = K\n",
    "    dict_['bi'] = bi.tolist()\n",
    "    dict_['bu'] = bu.tolist()\n",
    "    dict_['qi'] = qi.tolist()\n",
    "    dict_['pu'] = pu.tolist()\n",
    "    json_txt = json.dumps(dict_)\n",
    "    with open(filepath,'w') as file:\n",
    "        file.write(json_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(filepath):\n",
    "    with open(filepath,'r') as file:\n",
    "        dict_ = json.load(file)\n",
    "        \n",
    "        mu = dict_['mu']\n",
    "        K = dict_['K']\n",
    "        \n",
    "        bi = np.asarray(dict_['bi'])\n",
    "        bu = np.asarray(dict_['bu'])\n",
    "        \n",
    "        qi = np.asarray(dict_['qi'])\n",
    "        pu = np.asarray(dict_['pu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json('ml-100k/svd_model.json')\n",
    "load_json('ml-100k/svd_model.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对物品进行推荐\n",
    "\n",
    "def recommend(user):\n",
    "    cur_user_id = users_index[user]\n",
    "    \n",
    "    # 训练集中用户打过分的item\n",
    "    cur_user_items = user_items[cur_user_id]\n",
    "    \n",
    "    # 该用户对所有物品打分\n",
    "    user_item_scores = np.zeros(n_items)\n",
    "    \n",
    "    # 预测打分\n",
    "    for i in range(n_items):\n",
    "        if i not in cur_user_items:\n",
    "            user_item_scores[i] = svd_pred(cur_user_id,i)\n",
    "    # 用元组来存（分数，物品id）\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(user_item_scores))),reverse=True)\n",
    "    columns = ['item_id','score']\n",
    "    df = pd.DataFrame(columns=columns)\n",
    "    \n",
    "    for i in range(0,len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1]\n",
    "        # 把index转化乘list然后通过index定位value所在位置，然后再将key（物品真正的id）转化成list，找到真正的item id\n",
    "        cur_item = list(items_index.keys())[list(items_index.values()).index(cur_item_index)]\n",
    "        \n",
    "        if ~np.isnan(sort_index[i][0]) and cur_item_index not in cur_user_items:\n",
    "            df.loc[len(df)] = [cur_item,sort_index[i][0]]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>887431973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>875693118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>878542960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>874965706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>875073198</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp\n",
       "0        1        6       5  887431973\n",
       "1        1       10       3  875693118\n",
       "2        1       12       5  878542960\n",
       "3        1       14       5  874965706\n",
       "4        1       17       3  875073198"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "triplet_cols = ['user_id','item_id','rating','timestamp']\n",
    "df_triplet_test = pd.read_csv('ml-100k/u1.test',sep='\\t',encoding='Latin-1',names=triplet_cols)\n",
    "df_triplet_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "599 is new item. \n",
      "711 is new item. \n",
      "814 is new item. \n",
      "830 is new item. \n",
      "852 is new item. \n",
      "857 is new item. \n",
      "1156 is new item. \n",
      "1236 is new item. \n",
      "1309 is new item. \n",
      "1310 is new item. \n",
      "1320 is new item. \n",
      "1343 is new item. \n",
      "1348 is new item. \n",
      "1364 is new item. \n",
      "1373 is new item. \n",
      "1457 is new item. \n",
      "1458 is new item. \n",
      "1492 is new item. \n",
      "1493 is new item. \n",
      "1498 is new item. \n",
      "1505 is new item. \n",
      "1520 is new item. \n",
      "1533 is new item. \n",
      "1536 is new item. \n",
      "1543 is new item. \n",
      "1557 is new item. \n",
      "1561 is new item. \n",
      "1562 is new item. \n",
      "1563 is new item. \n",
      "1565 is new item. \n",
      "1582 is new item. \n",
      "1586 is new item. \n"
     ]
    }
   ],
   "source": [
    "# 统计总的用户\n",
    "unique_users_test = df_triplet_test['user_id'].unique()\n",
    "\n",
    "# 为每个用户推荐20个商品\n",
    "n_rec_items = 20\n",
    "\n",
    "#性能评价计算精确率和召回率\n",
    "n_hits = 0\n",
    "n_total_rec_items = 0\n",
    "n_test_items = 0\n",
    "\n",
    "#所有被推荐商品的集合，用于计算覆盖度\n",
    "all_rec_items = set()\n",
    "\n",
    "# 残差平方和，用于计算RMSE\n",
    "rss_test = 0.0\n",
    "\n",
    "# 对每个测试用户\n",
    "for user in unique_users_test:\n",
    "    if user not in users_index:\n",
    "        print('{} is new user'.format(user))\n",
    "        continue\n",
    "    user_records_test = df_triplet_test[df_triplet_test.user_id == user]\n",
    "    \n",
    "    rec_items = recommend(user)\n",
    "    \n",
    "    for i in range(n_rec_items):\n",
    "        item = rec_items.iloc[i]['item_id']\n",
    "        \n",
    "        if item in user_records_test['item_id'].values:\n",
    "            n_hits +=1\n",
    "            \n",
    "        all_rec_items.add(item)\n",
    "    \n",
    "    # 计算rmse\n",
    "    for i in range(user_records_test.shape[0]):\n",
    "        item = user_records_test.iloc[i]['item_id']\n",
    "        score = user_records_test.iloc[i]['rating']\n",
    "        \n",
    "        df1 = rec_items[rec_items.item_id == item]\n",
    "        if df1.shape[0] == 0:\n",
    "            print('{} is new item. '.format(item))\n",
    "            continue\n",
    "        pre_score = df1['score'].values[0]\n",
    "        rss_test += (pre_score - score)**2\n",
    "    # 推荐item 总数\n",
    "    n_total_rec_items += n_rec_items\n",
    "    n_test_items += user_records_test.shape[0]\n",
    "\n",
    "precision = n_hits / (1.0 * n_total_rec_items)\n",
    "recall = n_hits / (1.0 * n_test_items)\n",
    "\n",
    "# 覆盖率\n",
    "coverage = len(all_rec_items) / (1.0 * n_items)\n",
    "\n",
    "# 打分均方误差\n",
    "rmse = np.sqrt(rss_test/df_triplet_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0784313725490196"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.036"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1509090909090909"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9258171279468775"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
